package maumau;

import java.io.*;
import java.util.*;
import java.io.Serializable;

/**
 * @author Mathias Blaha
 * @author Maximilian Reß
 * @version 1.0
 */
public class MauMau_HAL9000 extends MauMau_Intf implements Serializable {

    private int numOfWonGames;
    private int action;
    private int tmpState;
    private int logGreedyAction = 0;
    private int logRandomAction = 0;
    private int logSumAction = 0;
    private long numOfPlayedEpisodes;
    private float epsilon = 0.5f;
    private Random rand;
    private Vector<Q> episodeQs;
    private Vector<Episode> episode;
    private HashMap<Integer, Vector<Q>> qsaMap;
    File trainedData = new File("HAL9000_trainingdata.ftw");
    File fnumOfPlayedEpisodes = new File("numOfPlayedEpisodes.txt");

    /*
     * C'tor
     */
    MauMau_HAL9000() {

        /*
         * Initialisierung
         */
        rand = new Random(0);
        numOfPlayedEpisodes = 0;
        numOfWonGames = 0;

        ObjectInputStream readTrainingData = null;
        ObjectOutputStream writeTrainingData = null;
        BufferedReader bReader = null;
        episode = new Vector<Episode>();

        /*
         * Laden der Trainingsdaten
         */

        try {
            readTrainingData = new ObjectInputStream(new FileInputStream(trainedData));
            qsaMap = (HashMap<Integer, Vector<Q>>) readTrainingData.readObject();
            readTrainingData.close();
        } catch (Exception e) {
            qsaMap = new HashMap<Integer, Vector<Q>>();
            System.err.println("Error loading " + trainedData + " --> creating new HashMap");

        }

        /*
         * Laden der Anzahl bereits gespielter Spiele, bezogen auf die
         * Trainingsdaten
         */

        try {
            bReader = new BufferedReader(new FileReader(fnumOfPlayedEpisodes));
            numOfPlayedEpisodes = Long.parseLong(bReader.readLine());
            bReader.close();
        } catch (Exception e) {
            System.err.println("Error loading " + fnumOfPlayedEpisodes);
        }

        /*
         * Einleitende Infos
         */

        System.out.println("Anzahl Spiele bisher:" + numOfPlayedEpisodes);
        System.out.println("Anzahl Spiele dieser Lauf:" + MauMau.numOfGames);
    }

    ;
        

    public void newGame() {
//        System.out.println(getName() + ": newGame called");
    }

    ;
    
/**
* Spielzug / Enthält gesamte Logik   
* @return (int) gewählte Aktion
*/
    
    
	@Override
    public int step(MauMau_State state, Vector<Integer> actions, double reward,
            boolean terminal) {


        /*
         * ENDE EINER EPISODE terminal == true Wichtig da Monte Carlo
         * Algorithmus, hier erfolgt das gesamte Lernen
         */

        if (terminal) {

            numOfPlayedEpisodes++;

            // Zwischenstand ausgeben nach 10% der Gesamtepisodenzahl:
//            if ((numOfPlayedEpisodes % (MauMau.numOfGames / 10)) == 0) {
//                System.out.println("Zwischenstand: " + numOfPlayedEpisodes);
//            }

            /*
             * positiver Return --> Episode / Spiel gewonnen
             */

            // have we won the game?
            if (reward > 0) {
                numOfWonGames++;
            }

            /*
             * Anpassen des Epsilon zum Sicherstellen von Exploration /
             * hardcoded
             */

            if (numOfPlayedEpisodes >= 10000000) {
                epsilon = 0.001f;
            } else if (numOfPlayedEpisodes >= 35000000) {
                epsilon = 0.005f;
            } else if (numOfPlayedEpisodes >= 2500000) {
                epsilon = 0.01f;
            } else if (numOfPlayedEpisodes >= 2000000) {
                epsilon = 0.1f;
            } else if (numOfPlayedEpisodes >= 1500000) {
                epsilon = 0.2f;
            } else if (numOfPlayedEpisodes >= 1000000) {
                epsilon = 0.3f;
            }

            /*
             * Update auf alle Q-Werte (episodeQs) in der Q(s,a), welche in der
             * letzten Episode (episode) besucht wurden
             */

            for (int i = 0; i < episode.size(); i++) {
                episodeQs = qsaMap.get(episode.get(i).episodeState);
                for (int j = 0; j < episodeQs.size(); j++) {
                    if (episodeQs.elementAt(j).action == episode.get(i).episodeAction) {
                        setReward(episodeQs.elementAt(j), reward);
                    }
                }
            }

            /*
             * Reset des Logs der States der vergangenen Episode
             */

            episode.clear();
            return 0;
        }


        /*
         * ENDE EINES SPIELZUGS Wählen einer geeigneten / gelernten Aktion oder
         * Erzeugne eines neuen States
         */


        /*
         * Generieren eines eindeutigen Schlüssels aus dem aktuellen
         * State-Objekt, dieser ist die Implementierung des "state"
         */
        //tmpState = hashCode(getCardColor(state.getOpenCard()), reduceStates(getCardType(state.getOpenCard())), state.getOpenCardMult(), actions);
        //tmpState = hashCode(getCardColor(state.getOpenCard()), reduceStates(getCardType(state.getOpenCard())), state.getOpenCardMult(), reduceActions(actions));
        tmpState = hashCode(state.getOpenCardMult(), actions);
        
        /*
         * State ist bereits bekannt --> Wählen einer Aktion
         */

        if (qsaMap.containsKey(tmpState)) {
            Vector<Q> q = qsaMap.get(tmpState);
            if (rand.nextFloat() < epsilon) {
                action = performRandomAction(actions);
            
            } else {
                int zuletztGewaehlt = 0;
                int greedyAction = q.elementAt(0).action;
                double tmpReward = q.elementAt(0).averageReward;
                for (int i = 1; i < q.size(); i++) {
                    if (q.elementAt(i).averageReward > tmpReward) {
                        greedyAction = q.elementAt(i).action;
                        tmpReward = q.elementAt(i).averageReward;
                        zuletztGewaehlt = i;
                    } else if (q.elementAt(i).averageReward == tmpReward) {
                        if (rand.nextFloat() >= 0.5f) {
                            greedyAction = q.elementAt(zuletztGewaehlt).action;
                            tmpReward = q.elementAt(zuletztGewaehlt).averageReward;
                            zuletztGewaehlt = i;
                        } else {
                            greedyAction = q.elementAt(i).action;
                            tmpReward = q.elementAt(i).averageReward;
                            zuletztGewaehlt = i;
                        }
                    }
                }
                action = greedyAction;
                logGreedyAction++;
            }

            /*
             * State ist noch unbekannt --> Anlegen des neuen States -->
             * zufällige Wahl einer möglichen Aktion
             */
        } else {

            /*
             * Erzeugen eines neuen States
             */
            Vector<Q> q = new Vector<Q>();
            for (int i = 0; i < actions.size(); i++) {
                q.add(new Q(actions.get(i)));
            }
            qsaMap.put(tmpState, q);
            action = performRandomAction(actions);
            // Alle Q(s,a) anlegen      

        }

        /*
         * Loggen der in diesem Zug gewählten (s,a)-Kombination
         */
        episode.add(new Episode(tmpState, action));
        return action;
    }

    /**
     * Wählt eine beliebige Aktion aus den in diesem Zug möglichen Aktionen
     * (actions)
     *
     * @param actions
     */
    private int performRandomAction(Vector<Integer> actions) {
        int pos = rand.nextInt(actions.size());
        Integer i = actions.get(pos);
        logRandomAction++;
        return i.intValue();
    }

    /**
     * Inkrementelles Update auf die Qs
     *
     * @param q
     * @param gainedReward
     */
    private void setReward(Q q, double gainedReward) {
        q.averageReward = q.averageReward + (gainedReward - q.averageReward) / (q.k + 1);
        q.k++;
    }

//        private void setReward(Q q, double gainedReward) {
//        q.averageReward = (q.averageReward * q.k + gainedReward)
//                / (q.k + 1);
//        q.k++;
//    }
    /**
     * Generieren eines eindeutigen Schlüssels, dieser ist die Implementierung
     * des "state" und dient als state-key für die HashMap Änderungen des state
     * durch hinzufügen weiterer Parameter
     *
     * @param openCard offen liegende Karte
     * @param openCardMult anzahl der 7er
     * @param actions mögliche Aktionen
     * @return
     */
//    private Vector<Integer> preHash(Integer topCardColor, Integer topCardType, Integer openCardMult, Vector<Integer> actions) {
//        Vector<Integer> prehash = new Vector<Integer>();
//        prehash.add(topCardColor);
//        prehash.add(topCardType);
//
//        if (openCardMult > 0) {
//            openCardMult = 43;
//        } else {
//            openCardMult = 44;
//        }
//
//        prehash.add(openCardMult);
//        Collections.sort(actions);
//        for (int i = 0; i < actions.size(); i++) {
//            prehash.add(actions.get(i));
//        }
//        prehash.trimToSize();
//        return prehash;
//    }

//    public int hashCode(int topCardColor, int topCardType, int openCardMult, Vector<Integer> actions) {
//        Collections.sort(actions);
//        if (openCardMult > 0) {
//            openCardMult = 43;
//        } else {
//            openCardMult = 44;
//        }
//        int hash = 5;
//        hash = 23 * hash + topCardColor;
//        hash = 23 * hash + topCardType;
//        hash = 23 * hash + openCardMult;
//        hash = 23 * hash + (actions != null ? actions.hashCode() : 0);
//        return hash;
//    }
    
       public int hashCode(int openCardMult, Vector<Integer> actions) {
        
           // to prevent multiple states for the same pool of actions, we need to sort them
           Collections.sort(actions);
        
           // we don't care if we have to draw twice or even more
           // we choose values which aren't used elsewhere
           if (openCardMult > 0) {
            openCardMult = 43;
        } else {
            openCardMult = 44;
        }
        
        // auto-generated stuff, we accept as it is
        int hash = 5;
        hash = 23 * hash + openCardMult;
        hash = 23 * hash + (actions != null ? actions.hashCode() : 0);
        return hash;
    }

    @Override
    public boolean equals(Object obj) {
        if (obj == null) {
            return false;
        }
        if (getClass() != obj.getClass()) {
            return false;
        }
        final MauMau_HAL9000 other = (MauMau_HAL9000) obj;
        return true;
    }

//    private String preHash(int topCardColor, int topCardType, int openCardMult, Vector<Integer> actions) {
//        String preHash = Integer.toString(topCardColor);
//      
//        preHash.concat(Integer.toString(topCardType));
//        preHash.concat(Integer.toString(openCardMult));
//        
//        Collections.sort(actions);
//        
//        for (int i = 0; i < actions.size(); i++) {
//            preHash.concat(Integer.toString(actions.get(i)));
//        }
//        
//        return preHash;
//    }
    
    // work's done! Now only a miracle can help us
    public void cleanup() {

//        System.out.println(getName() + ": Cleanup called");
//
//        /*
//         * Persistenz der Trainingsdaten
//         *
//         * Files im Workspace: trainedData.obj numOfPlayedEpisodes.txt
//         */
//
        ObjectOutputStream serializedHashMap = null;
        BufferedWriter bWriter = null;

        try {
            bWriter = new BufferedWriter(new FileWriter(fnumOfPlayedEpisodes));
            bWriter.write(Long.valueOf(numOfPlayedEpisodes).toString());
            serializedHashMap = new ObjectOutputStream(new FileOutputStream(trainedData));
            serializedHashMap.writeObject(qsaMap);
            serializedHashMap.flush();
            serializedHashMap.close();
            bWriter.flush();
            bWriter.close();
        } catch (Exception e) {
            System.out.println(e);
            System.err.println("error writing trainedData");
        }


        /*
         * Auswertung nach Ende aller Spiele
         */

        /**
         * Inhalte der qsa map
         */
//        for (Object values : qsaMap.values()){
//            System.out.println(values);
//        }
//        for (Vector<Integer> key : qsaMap.keySet()){
//            System.out.println(key);
//        }
        System.out.println("######################################");
        System.out.println("Anzahl Spiele gesamt: " + numOfPlayedEpisodes);
        System.out.println("Anzahl Spiele in diesem Lauf: " + MauMau.numOfGames);
        System.out.println("Anzahl gewonnener Spiele (): " + numOfWonGames);
        System.out.println("Gewonnen in %: " + (float) numOfWonGames / (float) MauMau.numOfGames * 100.0);
        System.out.println("Größe der qsa: " + qsaMap.size());
        System.out.println("Anzahl Greedy-Actions: " + logGreedyAction);
        System.out.println("Anzahl Random-Actions: " + logRandomAction);
        System.out.println("######################################");

    }

    /*
     * Datentyp für a,Q Paare bilden die Values in der HashMap qsa, zum key
     * stateHash (einem State)
     */
    private class Q implements Serializable {

        // Anzahl der Updates auf Q und wie oft Aktion a gespielt wurde
        int k;
        double averageReward;
        int action;

        Q(int action) {
            k = 0;
            this.averageReward = 0.0;
            this.action = action;
        }

        @Override
        public String toString() {
            String out = "| Action: " + this.action + "| Reward: " + this.averageReward + "| k: " + k;
            return out;
        }
    }

    /*
     * Datentyp zum sequentiellen Aufzeichnen der (state,action) Kombinationen
     * einer Episode
     */
    private class Episode {

        int episodeState;
        int episodeAction;

        public Episode(int stateHash, int action) {
            this.episodeState = stateHash;
            this.episodeAction = action;
        }
    }

    public int reduceStates(int topcard) {

        // pool cards with type NINE = 2, TEN = 3, DAME = 5, KÖNIG = 6, ASS = 7 in one proxy-state (42)
       if (topcard == MauMau_Constants.NEUN || topcard == MauMau_Constants.ZEHN || topcard == MauMau_Constants.DAME || topcard == MauMau_Constants.KOENIG || topcard == MauMau_Constants.ASS) {
            topcard = 42;
        }
        return topcard;
    }

    public Vector<Integer> reduceActions(Vector<Integer> actions) {

        TreeSet<Integer> reducedActions = new TreeSet<Integer>();
        Vector<Integer> pooledActions = new Vector<Integer>();

        for (int i = 0; i < actions.size(); i++) {
            if (getCardType(actions.get(i)) != MauMau_Constants.SIEBEN || getCardType(actions.get(i)) != MauMau_Constants.ACHT || actions.get(i) > MauMau_Constants.ASS) {
                reducedActions.add(getCardType(actions.get(i)));
                reducedActions.add(getCardColor(actions.get(i)));
            } else if (actions.get(i) == MauMau_Constants.BUBE_KREUZ_WISH_KREUZ || actions.get(i) == MauMau_Constants.BUBE_PIK_WISH_KREUZ || actions.get(i) == MauMau_Constants.BUBE_HERZ_WISH_KREUZ || actions.get(i) == MauMau_Constants.BUBE_KARO_WISH_KREUZ) {
                reducedActions.add(45); //wish KREUZ
            } else if (actions.get(i) == MauMau_Constants.BUBE_KREUZ_WISH_PIK || actions.get(i) == MauMau_Constants.BUBE_PIK_WISH_PIK || actions.get(i) == MauMau_Constants.BUBE_HERZ_WISH_HERZ || actions.get(i) == MauMau_Constants.BUBE_KARO_WISH_KARO) {
                reducedActions.add(46); //wish PIK
            } else if (actions.get(i) == MauMau_Constants.BUBE_KREUZ_WISH_HERZ || actions.get(i) == MauMau_Constants.BUBE_PIK_WISH_HERZ || actions.get(i) == MauMau_Constants.BUBE_HERZ_WISH_HERZ || actions.get(i) == MauMau_Constants.BUBE_KARO_WISH_HERZ) {
                reducedActions.add(47); //wish HERZ
            } else if (actions.get(i) == MauMau_Constants.BUBE_KREUZ_WISH_KARO || actions.get(i) == MauMau_Constants.BUBE_PIK_WISH_KARO || actions.get(i) == MauMau_Constants.BUBE_HERZ_WISH_KARO || actions.get(i) == MauMau_Constants.BUBE_KARO_WISH_KARO) {
                reducedActions.add(48); //wish KARO
            } else {
                reducedActions.add(actions.get(i));
            }
        }
                pooledActions.addAll(reducedActions);
                Collections.sort(pooledActions);
                pooledActions.trimToSize();        
        //doppelte werte reduzieren
        return pooledActions;
    }

    // Returns HERZ, PIK, KARO, KREUZ
    public int getCardColor(int k) {
        return k % 4;
    }

    // Cards 7,8,9,10,Bube,Dame,K�nig,Ass are numbered from 0 to 7
    // see NAMES[]
    public int getCardType(int k) {
        return k / 4;
    }
}
